# _*_coding:utf-8_*_

import requests
import time
import json
import re
import pymongo
from lxml import etree
from db import *

header = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "max-age=0",
    "Connection": "keep-alive",
    "Cookie": "UM_distinctid=1773e923249593-018e6758a55b39-326a7006-116d20-1773e92324a88d; taihe_bi_sdk_uid=79ebdef05b9148a04f20fc8dcab1fd37; taihe_bi_sdk_session=03833765bf2bff57c7eee58ac77a0831; ngacn0comUserInfo=kkkboking%09kkkboking%0939%0939%09%0910%090%094%090%090%09; ngaPassportUid=62881991; ngaPassportUrlencodedUname=kkkboking; ngaPassportCid=X93ruib6jkbp6a5uok4df470l7drab3emjg4slsi; ngacn0comUserInfoCheck=0aae10da5b17736ebd18e986fdbf730e; ngacn0comInfoCheckTime=1614584924; CNZZDATA30043604=cnzz_eid%3D413352232-1611659283-https%253A%252F%252Fbbs.nga.cn%252F%26ntime%3D1614652870; bbsmisccookies=%7B%22uisetting%22%3A%7B0%3A%22c%22%2C1%3A1614653734%7D%2C%22pv_count_for_insad%22%3A%7B0%3A-45%2C1%3A1614704504%7D%2C%22insad_views%22%3A%7B0%3A1%2C1%3A1614704504%7D%7D; _cnzz_CV30043604=forum%7Cfid706%7C0; lastvisit=1614653450; lastpath=/post.php",
    "Host": "bbs.nga.cn",
    "sec-ch-ua": '"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"',
    "sec-ch-ua-mobile": "?0",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "same-origin",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"
}
artictle = {"h": {"id": 30081858, "head": "helao_"}, "n": {"id": 24913158, "head": "niushen_"}, "hsz": {
    "id": 25909019, "head": ""}, "xin": {"id": 27239594, "head": "xin_"}, "z": {"id": 24914047, "head": "zz_"}, }

param_helao = {"tid": 30081858, "authorid": 5254815,
               "man": "禾佬", "key": "helao"}
param_lang = {"tid": 30081858, "authorid": 150058, "man": "狼经理", "key": "lang"}
param_gui = {"tid": 30081858, "authorid": 2251556, "man": "龟哥", "key": "gui"}
param_fred = {"tid": 30081858, "authorid": 34795211,
              "man": "fred1024", "key": "fred"}
param_htg = {"tid": 30081858, "authorid": 60259365, "man": "黄图哥", "key": "htg"}
param_yo = {"tid": 30081858, "authorid": 38574993, "man": "YOYO", "key": "yo"}
param_xin = {"tid": 30081858, "authorid": 614111,
             "man": "XINQING", "key": "xin"}
param_qq = {"tid": 30081858, "authorid": 9439143, "man": "假QQ", "key": "qq"}
param_haohan = {"tid": 30081858, "authorid": 40810297,
                "man": "好汉歌", "key": "haohan"}

param_niu = {"tid": 30081858, "authorid": 4627122, "man": "牛神", "key": "niu"}
param_sun = {"tid": 30081858, "authorid": 37876, "man": "孙哥", "key": "sun"}
param_pcg = {"tid": 30081858, "authorid": 43299140, "man": "PC哥", "key": "pcg"}
param_zqq = {"tid": 30081858, "authorid": 11510983,
             "man": "真QQ哥", "key": "zqq"}

paramHSZ = {"tid": 25909019, "authorid": 62719274, "man": "黑水住", "key": "hsz"}

paramZZ = {"tid": 24914047, "authorid": 61510173,
           "man": "ZZonline", "key": "zzonline"}

url1 = "https://bbs.nga.cn/"
# end = 1


def get_page(session, url, header):
    resp = session.get(url=url, headers=header)
    if resp.status_code != 200:
        headers = {
            'Content-Type': 'application/json'
        }
        ding_url1 = "https://oapi.dingtalk.com/robot/send?access_token=354c22f482d29583bdf4636438cda158b8acd5686f42b0929277d933dd95dd49"
        mg = {
            "msgtype": "markdown",
            "markdown": {
                "title": "发言",
                "text": "#### 禾佬发言 \n> cookies 失效了\n"
            },
            "at": {
                "atMobiles": [],
                "isAtAll": False
            }
        }
        # requests.post(ding_url1, data=json.dumps(mg),headers=headers)
    print(resp.status_code)
    return resp.text


def parse(html, authorid):
    html = etree.HTML(html)
    html_data = html.xpath(
        '//html/body/div[@id="mmc"]/div[@id="mc"]/div[@id="m_posts"]/div[@id="m_posts_c"]/table/tr/td[@class="c2"]')
    l = []
    ids = []
    for x in html_data:
        ret = {}
        uid = x.xpath('./a/@id')
        ts = x.xpath('./div/span[@title="reply time"]/text()')
        content = x.xpath('./span/span')
        if len(content) != 1:
            continue
        content2 = etree.tostring(content[0], encoding='utf-8')
        ret["_id"] = uid[0] + str(authorid)
        ret["time"] = ts[0]
        ret["content"] = str(content2, encoding='utf-8')
        ret["send"] = 1
        l.append(ret)
        ids.append(uid[0])
        # print(type(content2))
        # print(uid)
        # print(ts)
        # print(str(content2, encoding='utf-8'))

    return l, ids


def get_set_page(html, num):
    html = etree.HTML(html)
    page = html.xpath(
        '//html/body/div[@id="mmc"]/div[@id="mc"]/div[@id="m_pbtnbtm"]/div/div[@class="left"]/a[@title="下一页"]/@href')
    if len(page) == 1:
        page = page[0]
        num = int(page.split("page=")[-1]) - 1
        return True, page, num
    else:
        page = html.xpath(
            '//html/body/div[@id="mmc"]/div[@id="mc"]/div[@id="m_pbtnbtm"]/div/div[@class="left"]/a[@title="上一页"]/@href')
        if len(page) == 1:
            page = page[0]
            num = int(page.split("page=")[-1]) + 1
        return False, page, num


def start(db, url1, end, p, page_num, authorid):
    session = requests.session()
    page1 = get_page(session, url1 + end % (p, page_num), header)
    flag, next_url, num = get_set_page(page1, page_num)
    ret, ids = parse(page1, authorid)
    for i in ret:
        info = db.update(i, {"$setOnInsert": i}, upsert=True)
    # db.insert_many(ret)

    while flag:
        time.sleep(1)
        page1 = get_page(session, url1 + next_url, header)
        flag, next_url, num = get_set_page(page1, page_num)
        ret, ids = parse(page1, authorid)
        for i in ret:
            db.update(i, {"$setOnInsert": i}, upsert=True)
    return num


def mongo():
    myclient = pymongo.MongoClient("mongodb://localhost:7891/")
    db = myclient.admin
    # 认证
    db.authenticate('adminkpmg', 'kpmg123')
    mydb = myclient["spider"]
    mycol = mydb["msg"]
    return mycol


def msg(db, url1, end, tid, authorid, page_num, man):
    end = "read.php?tid=%s&authorid=%s&page=%s"
    session = requests.session()
    print(url1 + end % (tid, authorid, page_num), man)
    page1 = get_page(session, url1 + end % (tid, authorid, page_num), header)
    msg_list = []
    if '找不到内容 或 没有更多页了' in page1:
        return 1, []
    flag, next_url, num = get_set_page(page1, page_num)
    ret, ids = parse(page1, authorid)
    for i in ret:
        get_message(i, man)
        try:
            info = db.update(i, {"$setOnInsert": i}, upsert=True)
            if not info["updatedExisting"]:
                msg_list.append(i)
        except Exception as e:
            p = re.search('"(.*?)"', str(e))
            if p:
                name = p.groups()[0]
                db.delete_many({"_id": name})
            info = db.update(i, {"$setOnInsert": i}, upsert=True)
            if not info["updatedExisting"]:
                msg_list.append(i)

    # db.insert_many(ret)

    while flag:
        time.sleep(1)
        page1 = get_page(session, url1 + next_url, header)
        flag, next_url, num = get_set_page(page1, page_num)
        ret, ids = parse(page1, authorid)
        for i in ret:
            get_message(i, man)
            try:
                info = db.update(i, {"$setOnInsert": i}, upsert=True)
                if not info["updatedExisting"]:
                    msg_list.append(i)
            except Exception as e:
                p = re.search('"(.*?)"', str(e))
                if p:
                    name = p.groups()[0]
                    db.delete_many({"_id": name})
                info = db.update(i, {"$setOnInsert": i}, upsert=True)
                if not info["updatedExisting"]:
                    msg_list.append(i)
    return num, msg_list


def get_message(i, man):
    pt = "<.*?>"
    s = re.sub(pt, "", i["content"])
    s = s.replace("[quote]", "")
    if "[/quote]" in s:
        s = s.split("[/quote]")
        s1 = '#### {}发言 \n> {} \n'.format(man, s[-1].strip())
        s2 = s[0]
        if 'Post by' and '[/uid]' in s2:
            p = re.search("Post by(.*?)\[\/uid\]", s2)
            if p:
                name = p.groups()[0]
                c = s2.split("[/uid]")[-1]
                c = c.replace(":[/b]", ': ')
                c = c.replace("(", '')
                c = c.replace(")", '')
                s2 = '#### {}发言 \n> {}\n'.format(name.strip(), c.strip())
        s = s2+s1
    else:
        s = '#### {}发言 \n> {} \n'.format(man, s)
    s = re.sub(
        "\[img\]\.", "\n> ![screenshot](https://img.nga.178.com/attachments", s)
    s = s.replace("[/img]", ")\n> ")
    i["content"] = s


def send_msg(msg_list, mg, ding_url, headers, man):
    print(len(msg_list))
    for i in msg_list:
        # pt = "<.*?>"
        # s = re.sub(pt, "", i["content"])
        # s = s.replace("[quote]", "")
        # if "[/quote]" in s:
        #     s = s.split("[/quote]")
        #     s1 = '#### {}发言 \n> {} \n'.format(man, s[-1].strip())
        #     s2 = s[0]
        #     if 'Post by' and '[/uid]' in s2:
        #         p = re.search("Post by(.*?)\[\/uid\]", s2)
        #         if p:
        #             name = p.groups()[0]
        #             c = s2.split("[/uid]")[-1]
        #             c = c.replace(":[/b]", ': ')
        #             c = c.replace("(", '')
        #             c = c.replace(")", '')
        #             s2 = '#### {}发言 \n> {}\n'.format(name.strip(), c.strip())
        #     s = s2+s1
        # else:
        #     s = '#### {}发言 \n> {} \n'.format(man, s)
        # s = re.sub("\[img\]\.", "\n> ![screenshot](https://img.nga.178.com/attachments", s)
        # s = s.replace("[/img]", ")\n> ")
        mg["markdown"]["text"] = i["content"] + i["time"]
        # time.sleep(0.5)
        # print(i["content"] + i["time"])
        requests.post(ding_url, data=json.dumps(mg), headers=headers)
# mission()


def stand_v2():
    print("v2")
    ding_url1 = "https://oapi.dingtalk.com/robot/send?access_token=354c22f482d29583bdf4636438cda158b8acd5686f42b0929277d933dd95dd49"

    ding_url4 = "https://oapi.dingtalk.com/robot/send?access_token=e5225fccf7f2fafee606d62262a61b9b632bec27d7708ad82a28292bc4fdef55"

    ding_url8 = "https://oapi.dingtalk.com/robot/send?access_token=a84b0fbddc6bc69d4f91b5d3e631cc53b18331e3612d64788b870523832b747c"

    ding_url9 = "https://oapi.dingtalk.com/robot/send?access_token=6d9c15f33519793287e97bef22403dc1875620fc3bcce9a73309ca3f84f11552"
    ding_url10 = "https://oapi.dingtalk.com/robot/send?access_token=2b686334ae2163c11560acab35aa98dd2007004f746c07cc8d23e44fe6010d79"
    mg = {
        "msgtype": "markdown",
        "markdown": {
            "title": "发言",
            "text": "#### NGAxx发言 \n> 9度，西北风1级，空气良89，相对温度73%\n #### 禾佬发言 \n> 10点20分发布 [天气](https://www.dingtalk.com) \n"
        },
        "at": {
            "atMobiles": [],
            "isAtAll": False
        }
    }
    headers = {
        'Content-Type': 'application/json'
    }
    # requests.post(ding_url, data=json.dumps(mg),headers=headers)

    d = settings_db.find_one({"name": "page_number"})
    mdb = mongo()
    big(mdb, mg, ding_url1, headers, d, param_helao, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_lang, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_gui, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_fred, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_htg, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_xin, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_yo, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_qq, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_niu, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_sun, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_pcg, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_zqq, artictle['h'])
    big(mdb, mg, ding_url1, headers, d, param_haohan, artictle['h'])

   
    big(mdb, mg, ding_url4, headers, d, paramZZ, artictle['z'])
    print(d)
    settings_db.update_one({"name": "page_number"}, {"$set": d}, upsert=True)


def test():
    ding_url1 = "https://oapi.dingtalk.com/robot/send?access_token=354c22f482d29583bdf4636438cda158b8acd5686f42b0929277d933dd95dd49"

    mg = {
        "msgtype": "markdown",
        "markdown": {
            "title": "发言",
            "text": "#### [uid=42371166]花空少年发言 \n> 2021-03-08 15:06: [s:ac:呆]…… ![screenshot](https://img.nga.178.com/attachments/mon_202103/08/jmQj0l-ayalK1vT1kSg0-sg.jpg.medium.jpg) \n #### 禾佬发言 \n> 10点20分发布 [天气](https://www.dingtalk.com) \n"
        },
        "at": {
            "atMobiles": [],
            "isAtAll": False
        }
    }
    headers = {
        'Content-Type': 'application/json'
    }
    requests.post(ding_url1, data=json.dumps(mg), headers=headers)


# test()

# stand()

def big(mdb, mg, ding_url1, headers, d, param, artictle):
    try:
        end = 1
        print(param["man"])
        mg["markdown"]["title"] = "%s发言" % param["man"]
        key = artictle["head"] + param["key"]
        numH, msg_list1 = msg(
            mdb, url1, end, artictle["id"], param["authorid"], d[key], param["man"])
        send_msg(msg_list1, mg, ding_url1, headers, param["man"])
        d[key] = numH
    except Exception as e:
        print(e)
        mg["markdown"]["text"] = "%sbug" % param["man"]
        requests.post(url1, data=json.dumps(mg), headers=headers)


# stand_v2()
# z = {'helao_helao': 90, 'helao_lang': 380, 'helao_gui': 41, 'helao_htg': 219, 'helao_fred': 113, 'helao_xin': 31, 'helao_yo': 36, 'helao_qq': 29, 'hsz': 34, 'helao_zqq': 40, 'helao_pcg': 67, 'helao_niu': 42, 'helao_sun': 8, 'helao_haohan': 88, 'niushen_niu': 135, 'niushen_sun': 46, 'niushen_pcg': 166, 'niushen_lang': 26, 'niushen_qq': 224, 'xin_xin': 37}
# with open('/home/yk/kapp/spider_nga/num.json', 'w') as f:
#         json.dump(z, f)
